package emissary.output;

import emissary.config.ConfigUtil;
import emissary.config.Configurator;
import emissary.core.Family;
import emissary.core.IBaseDataObject;
import emissary.util.ShortNameComparator;
import emissary.util.TimeUtil;
import emissary.util.shell.Executrix;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.SecureRandom;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SimpleTimeZone;
import java.util.UUID;
import javax.annotation.Nullable;

import static emissary.core.Form.PREFIXES_LANG;
import static emissary.core.Form.TEXT;
import static emissary.core.Form.UNKNOWN;
import static emissary.core.constants.Parameters.ORIGINAL_FILENAME;

public class DropOffUtil {
    protected static final Logger logger = LoggerFactory.getLogger(DropOffUtil.class);

    protected static final String SEPARATOR = FileSystems.getDefault().getSeparator();
    protected static final String OS_NAME = System.getProperty("os.name").toUpperCase();

    protected String unixRoot;

    protected String placeOutputData;

    /** Sources for building an ID for an item */
    protected List<String> idTokens = new ArrayList<>();

    /** Sources for building a date string for an item */
    protected List<String> dateTokens = new ArrayList<>();

    /** params to get from parent and save as PARENT_param */
    protected List<String> parentParams = new ArrayList<>();

    /** Output directories */
    protected Map<String, File> outputDirectories = new HashMap<>();

    protected Executrix executrix;

    // Items for generating random filenames
    protected static SecureRandom prng = new SecureRandom();
    protected static final SimpleDateFormat DATE_PATTERN = new SimpleDateFormat("yyyyDDDHHmmss");
    protected static final byte[] ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".getBytes();
    protected static String prefix = "TXT";
    protected boolean uuidInOutputFilenames = true;
    protected int maxFilextLen = Integer.MAX_VALUE;

    // Items for generating UUIDs
    private static final String AUTO_GENERATED_ID = "AUTO_GENERATED_ID";
    private static final String PARENT_AUTO_GENERATED_ID = "PARENT_AUTO_GENERATED_ID";
    private String autoGeneratedIdPrefix = null;

    private static final String SHORTNAME = "SHORTNAME";

    private static final String DEFAULT_EVENT_DATE_TO_NOW = "DEFAULT_EVENT_DATE_TO_NOW";
    protected boolean defaultEventDateToNow = true;

    /**
     * Create with the default configuration
     */
    public DropOffUtil() {
        configure(null);
    }

    /**
     * Create with the specified configuration
     */
    public DropOffUtil(final Configurator configG) {
        configure(configG);
    }

    /**
     * Set up config for this class
     * <ul>
     * <li>ID_PARAMETER : multiple parameter values, ordered list of how to build a EMISSARY_ID</li>
     * <li>ID : backwards compatibility for ID_PARAMETER only used if ID_PARAMETER does not exist</li>
     * <li>DATE_PARAMETER : multiple parameter values, ordered list of how to build a date path</li>
     * <li>DATE_FORMAT: string to use for SimpleDateFormat, ordered list</li>
     * <li>OUTPUT_FILE_PREFIX: string to use when generating random filenames, dflt: TXT</li>
     * <li>UUID_IN_OUTPUT_FILENAMES: boolean [true]</li>
     * <li>AUTO_GENERATED_ID_PREFIX: prefix to use for an auto-generated id</li>
     * </ul>
     */
    protected void configure(final Configurator configG) {
        Configurator actualConfigG = configG;
        if (actualConfigG == null) {
            try {
                actualConfigG = ConfigUtil.getConfigInfo(DropOffUtil.class);
            } catch (IOException e) {
                logger.error("Cannot open default config file", e);
            }
        }

        if (actualConfigG != null) {
            this.placeOutputData = actualConfigG.findStringEntry("OUTPUT_DATA", "outputData");
            this.unixRoot = actualConfigG.findStringEntry("UNIX_ROOT", null);
            this.executrix = new Executrix(actualConfigG);
            this.idTokens = actualConfigG.findEntries("ID_PARAMETER");
            this.autoGeneratedIdPrefix = actualConfigG.findStringEntry("AUTO_GENERATED_ID_PREFIX");

            // truncate the prefix if necessary
            if (!StringUtils.isBlank(this.autoGeneratedIdPrefix) && (this.autoGeneratedIdPrefix.length() > 4)) {
                this.autoGeneratedIdPrefix = this.autoGeneratedIdPrefix.substring(0, 4);
            }
            if (this.idTokens.size() == 0) {
                this.idTokens = actualConfigG.findEntries("ID");
            }
            this.dateTokens = actualConfigG.findEntries("DATE_PARAMETER");
            this.parentParams = actualConfigG.findEntries("PARENT_PARAM");


            this.defaultEventDateToNow = actualConfigG.findBooleanEntry(DEFAULT_EVENT_DATE_TO_NOW, this.defaultEventDateToNow);

            prefix = actualConfigG.findStringEntry("OUTPUT_FILE_PREFIX", prefix);
            this.uuidInOutputFilenames = actualConfigG.findBooleanEntry("UUID_IN_OUTPUT_FILENAMES", this.uuidInOutputFilenames);
            // limit the length of the FILEXT param
            this.maxFilextLen = actualConfigG.findIntEntry("MAX_FILEXT_LEN", this.maxFilextLen);
            if (this.maxFilextLen < 0) {
                this.maxFilextLen = Integer.MAX_VALUE;
            }
        } else {
            logger.debug("Configuration is null for DropOffUtil, using defaults");
            this.executrix = new Executrix();
        }
    }

    /**
     * Generate a new random build file name using the configured prefix and stragety
     */
    public String generateBuildFileName() {
        if (this.uuidInOutputFilenames) {
            return (prefix + getDate(new Date()) + UUID.randomUUID());
        } else {
            // Using some constants plus yyyyJJJhhmmss plus random digit,
            // letter, digit
            return (prefix + getDate(new Date()) + prng.nextInt(10) + ALPHABET[prng.nextInt(ALPHABET.length)] + prng.nextInt(10));
        }
    }

    private static String getDate(final Date d) {
        synchronized (DATE_PATTERN) {
            return DATE_PATTERN.format(d);
        }
    }

    /**
     * Drop off often needs to make way for a file it wants to write Do so and return the name of the file that can now be
     * written
     */
    public String makeWayForIncomingFile(final IBaseDataObject DO, final String suffix) {
        return makeWayForIncomingFile(DO, suffix, null);
    }

    public String makeWayForIncomingFile(final IBaseDataObject DO, final String suffix, final String spec) {
        // Construct output file from DropOff output path and IBaseDataObject
        // filename
        final String outputFile = getShortOutputFileName(DO, spec);

        // Return value when finally set
        final String fileName = outputFile + suffix;

        // Set up path and make it writable
        if (!setupPath(fileName)) {
            return null;
        }

        if (removeExistingFile(fileName)) {
            return fileName;
        }

        return null;
    }

    /**
     * Remove a file if it exists
     */
    public boolean removeExistingFile(final String fileName) {
        // If a file already exists under this name, we're going
        // to have to move it aside. This is going to break the link
        // if it is an attachment to another parent document.
        final Path theFile = Paths.get(fileName);
        try {
            Files.deleteIfExists(theFile);
        } catch (IOException e) {
            logger.error("Trouble removing file: {}", theFile, e);
        }
        return !Files.exists(theFile);
    }

    /**
     * mkdir -p to a path and make sure it is writable
     *
     * @param fileName the file name, including directory and filename parts
     * @return true iff it works
     */
    public boolean setupPath(final String fileName) {
        final String pathName = fileName.substring(0, fileName.lastIndexOf(SEPARATOR));
        final Path thePath = Paths.get(pathName);

        // If the specified output directory doesn't exist try creating it
        if (!Files.exists(thePath)) {
            int tryCount = 1;
            do {
                try {
                    Files.createDirectories(thePath);
                } catch (IOException e) {
                    logger.warn("Trouble setting up directories:{}", thePath, e);
                }
                if (!Files.exists(thePath)) {
                    try {
                        Thread.sleep(50L * tryCount);
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                    }
                }
                tryCount++;
            } while (!Files.exists(thePath) && tryCount <= 10);

            if (!Files.exists(thePath)) {
                logger.warn("Cannot create directory for output: {} in {} attempts", thePath, tryCount);
                return false;
            }

            if (tryCount > 2 && logger.isDebugEnabled()) {
                logger.debug("Output path created for {} but it took {} attempts", thePath, tryCount);
            }
        }

        // If the specified output directory doesn't have write
        // permission try to fix it
        if (!Files.isWritable(thePath)) {
            logger.warn("Dont have write permission for {}, setting it now", pathName);
            if (!thePath.toFile().setWritable(true) || !Files.isWritable(thePath)) {
                logger.warn("Cannot write to directory for output: {}", thePath);
                return false;
            }
        }

        return true;
    }

    public String getOutputDirectory() {
        return this.placeOutputData;
    }

    /**
     * Create a path name from the spec for the specified spec using d as the TLD if d is a TLD, null otherwise
     *
     * @param spec the spec to fill in
     * @param d the payload to pull values from
     * @see #getPathFromSpec(String,IBaseDataObject,IBaseDataObject)
     */
    public String getPathFromSpec(final String spec, final IBaseDataObject d) {
        // pass self as tld if tld, null if not
        return getPathFromSpec(spec, d, (d != null && d.shortName().indexOf(emissary.core.Family.SEP) == -1) ? d : null);
    }

    /**
     * Create a path name from the spec for the specified spec Specs understand the following 'language' of replacement
     * stuff. Anything not understood is a literal
     *
     * <pre>
     * <code>@TLD{'KEY'}</code> is to pull the named KEY from the top level document Metadata
     * <code>@META{'KEY'}</code> is to pull the named KEY from the MetaData
     * %U% = USER
     * %I% = INPUT_FILE_NAME (whole thing)
     * %S% = INPUT_FILE SHORT NAME
     * %P% = INPUT_FILE PATH (all but short name)
     * %i% = INPUT_FILE_NAME with slashes converted to underscores
     * %p% = INPUT_FILE PATH with slashes converted to underscores
     * %F% = FILETYPE
     * %L% = LANGUAGE
     * %G% = DTG multi directory layout yyyy-mm-dd/hh/mi(div)10
     * %R% = ROOT (Unix or Win depending on OS)
     * %B% = ID for the payload depending on type (no -att-)
     * %b% = ID for the payload depending on type (with -att-)
     * %Y% = Four digit year
     * %M% = Two digit month
     * %D% = Two digit day of month
     * %J% = Three digit ordinal day of the year
     * </pre>
     *
     * @param specArg the incoming specification
     * @param d the payload we are making a path for
     * @param tld the top level document in the d family, possibly null
     * @return string path name with correct separators for this OS
     */
    public String getPathFromSpec(final String specArg, @Nullable final IBaseDataObject d, @Nullable final IBaseDataObject tld) {
        final StringBuffer sb = new StringBuffer(128);

        // Provide a default spec, just like the old days...
        String spec = specArg;
        if (spec == null) {
            spec = "%R%/@TLD{'TARGETBIN'}/%S%";
        }

        for (int i = 0; i < spec.length(); i++) {
            final char c = spec.charAt(i);

            if (c == '%' && i < spec.length() - 2) {
                final char t = spec.charAt(i + 1);
                final char x = spec.charAt(i + 2);

                if (x == c) {
                    switch (t) {
                        case 'U':
                            if (tld != null) {
                                sb.append(nvl(tld.getParameter("UserName"), "no-userid"));
                            } else if (d != null) {
                                sb.append(nvl(d.getParameter("UserName"), "no-userid"));
                            }
                            break;
                        case 'S':
                            if (d != null) {
                                sb.append(d.shortName());
                            }
                            break;
                        case 'I':
                            if (d != null) {
                                sb.append(d.getFilename());
                            }
                            break;
                        case 'i':
                            if (d != null) {
                                sb.append(d.getFilename().replaceAll("[/\\\\]", "_"));
                            }
                            break;
                        case 'P':
                            if (d != null) {
                                sb.append(d.getFilename().substring(0, d.getFilename().length() - d.shortName().length()));
                            }
                            break;
                        case 'p':
                            if (d != null) {
                                sb.append(d.getFilename().substring(0, d.getFilename().length() - d.shortName().length()).replaceAll("[/\\\\]", "_"));
                            }
                            break;
                        case 'F':
                            if (d != null) {
                                sb.append(nvl(cleanSpecPath(d.getFileType()), "NONE"));
                            }
                            break;
                        case 'L':
                            if (d != null) {
                                sb.append(nvl(d.getParameter("LANGUAGE"), "NONE"));
                            }
                            break;
                        case 'G':
                            if (tld != null) {
                                sb.append(datePath(cleanSpecPath(tld.getStringParameter("DTG"))));
                            } else if (d != null) {
                                sb.append(datePath(cleanSpecPath(d.getStringParameter("DTG"))));
                            }
                            break;
                        case 'R':
                            sb.append(getRootPath());
                            break;
                        case 'B':
                            if (tld != null) {
                                sb.append(cleanSpecPath(getBestIdFrom(tld)));
                            } else if (d != null) {
                                sb.append(cleanSpecPath(getBestIdFrom(d)));
                            }
                            break;
                        case 'b':
                            sb.append(cleanSpecPath((tld != null) ? getBestIdFrom(tld) : getBestIdFrom(d)));
                            final String sn = d.shortName();
                            final int pos = sn.indexOf(emissary.core.Family.SEP);
                            if (pos > 0) {
                                sb.append(sn.substring(pos));
                            }
                            break;
                        case 'Y':
                            sb.append(TimeUtil.getDate("yyyy", "GMT"));
                            break;
                        case 'M':
                            sb.append(TimeUtil.getDate("MM", "GMT"));
                            break;
                        case 'D':
                            sb.append(TimeUtil.getDate("dd", "GMT"));
                            break;
                        case 'J':
                            sb.append(TimeUtil.getDate("DDD", "GMT"));
                            break;
                        default:
                            sb.append(c).append(t).append(x);
                    }
                    i += 2; // SUPPRESS CHECKSTYLE ModifiedControlVariable
                } else {
                    // No trailing % after character
                    sb.append(c);
                }

            } else if (c == '@' && i < spec.length() - 8 && spec.charAt(i + 1) == 'M' && spec.charAt(i + 2) == 'E' && spec.charAt(i + 3) == 'T'
                    && spec.charAt(i + 4) == 'A' && spec.charAt(i + 5) == '{' && spec.charAt(i + 6) == '\'') {

                final int endpos = spec.indexOf("'", i + 7);
                if (endpos > i + 7) {
                    final String token = spec.substring(i + 7, endpos);
                    final String value = cleanSpecPath(d.getStringParameter(token));
                    sb.append(nvl(value, "NO-" + token));
                    i += 8 + token.length(); // META{'token'} SUPPRESS CHECKSTYLE ModifiedControlVariable
                } else {
                    sb.append(c);
                }
            } else if (c == '@' && i < spec.length() - 7 && spec.charAt(i + 1) == 'T' && spec.charAt(i + 2) == 'L' && spec.charAt(i + 3) == 'D'
                    && spec.charAt(i + 4) == '{' && spec.charAt(i + 5) == '\'' && tld != null) {
                final int endpos = spec.indexOf("'", i + 6);
                if (endpos > i + 6) {
                    final String token = spec.substring(i + 6, endpos);
                    final String value = cleanSpecPath(tld.getStringParameter(token));
                    sb.append(nvl(value, "NO-" + token));
                    i += 7 + token.length(); // TLD{'token'} SUPPRESS CHECKSTYLE ModifiedControlVariable
                } else {
                    sb.append(c);
                }
            } else {
                sb.append(c);
            }

        }

        String answer = sb.toString();

        // Set the proper path separator
        answer = answer.replace('\\', '/');
        answer = answer.replaceAll("\\.([/\\\\])", "_$1");

        return answer;
    }

    protected String cleanSpecPath(@Nullable String token) {
        return token == null ? null : token.replaceAll("[.]+", ".");
    }

    /**
     * Extract the ID from the payload. The ID from the payload is specified in the cfg file. An ID = SHORTNAME will use the
     * shortname. An ID = AUTO_GENERATED_ID will use an auto gen uuid. If no id value is found, defaults to using an auto
     * generated id.
     *
     * @return id
     */
    public String getBestIdFrom(final IBaseDataObject d) {
        for (final String s : this.idTokens) {
            if (!StringUtils.isBlank(d.getStringParameter(s))) {
                return d.getStringParameter(s);
            }
            if (SHORTNAME.equals(s)) {
                final String shortName = d.shortName();
                // if shortname is not blank use it, if blank move on...
                if (!StringUtils.isBlank(shortName)) {
                    return shortName;
                }
            }
            if (AUTO_GENERATED_ID.equals(s)) {
                return getRandomUUID(d);
            }
        }
        // if nothing else worked, use an auto gen id
        return getRandomUUID(d);
    }

    /**
     * Extract the ID from the payload. The ID from the payload is specified in the cfg file. An ID = SHORTNAME will use the
     * shortname. An ID = AUTO_GENERATED_ID will be ignored. If no id value is found, returns empty array.
     *
     * @return id
     */
    public String[] getExistingIds(final IBaseDataObject d) {
        final List<String> values = new ArrayList<>();
        for (final String s : this.idTokens) {
            if (!StringUtils.isBlank(d.getStringParameter(s))) {
                values.add(d.getStringParameter(s));
            }
            if (SHORTNAME.equals(s)) {
                final String shortName = d.shortName();
                // if shortname is not blank use it, if blank move on...
                if (!StringUtils.isBlank(shortName)) {
                    values.add(shortName);
                }
            }
        }
        return values.toArray(new String[0]);
    }

    public String getRootPath() {
        return this.unixRoot;
    }

    public String getSubDirName(final IBaseDataObject d) {
        return getSubDirName(d, null, null);
    }

    public String getSubDirName(final IBaseDataObject d, @Nullable final String spec, final IBaseDataObject tld) {
        String fileName = null;

        if (spec != null && spec.length() > 0) {
            fileName = getPathFromSpec(spec, d);
        }

        if (fileName != null && fileName.length() > 0) {
            logger.debug("usingPathFromSpec instead of TARGETBIN: {}", fileName);
            return fileName;
        } else if (tld.getStringParameter("TARGETBIN") != null) {
            logger.debug("TARGETBIN is " + tld.getParameter("TARGETBIN"));
            return fixFileNameSeparators(tld.getStringParameter("TARGETBIN"));
        } else {
            logger.debug("TARGETBIN is null");
            return whatBin(null);
        }
    }

    /**
     * This is a fallback, we should have either the bin name from the plop line or the processing time plugged in by pickup
     * but if not, we have to send it somewhere...
     */
    public String whatBin(final String sriheader) {
        final SimpleTimeZone gmt = new SimpleTimeZone(0, "GMT");
        final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd" + SEPARATOR + "HH" + SEPARATOR + "mm");
        sdf.setTimeZone(gmt);
        final Date currentTime_1 = new Date();
        final String dateStringDefault = sdf.format(currentTime_1);
        return ("NO-CASE" + SEPARATOR + dateStringDefault);
    }

    public String getRelativeShortOutputFileName(final IBaseDataObject d) {
        return getRelativeShortOutputFileName(d, null);
    }

    public String getRelativeShortOutputFileName(final IBaseDataObject d, final String spec) {
        return getRelativeShortOutputFileName(d, spec, d.shortName().indexOf(emissary.core.Family.SEP) == -1 ? d : null);
    }

    public String getRelativeShortOutputFileName(final IBaseDataObject d, final String spec, final IBaseDataObject tld) {
        final String sdir = getSubDirName(d, spec, tld);
        if ("".equals(sdir)) {
            return SEPARATOR + d.shortName();
        } else {
            return SEPARATOR + sdir + SEPARATOR + d.shortName();
        }
    }

    public String getShortOutputFileName(final IBaseDataObject d) {
        return getShortOutputFileName(d, null);
    }

    public String getShortOutputFileName(final IBaseDataObject d, final String spec) {
        return getShortOutputFileName(d, spec, d.shortName().indexOf(emissary.core.Family.SEP) == -1 ? d : null);
    }

    public String getShortOutputFileName(final IBaseDataObject d, final String spec, final IBaseDataObject tld) {
        return getRelativeShortOutputFileName(d, spec, tld);
    }

    /**
     * Replace any file separators that are not for this platform with the correct one
     */
    public String fixFileNameSeparators(final String s) {
        // other platform;
        String badfs = null;

        if ("/".equals(SEPARATOR)) {
            badfs = "\\";
        } else {
            badfs = "/";
        }

        final StringBuilder ret = new StringBuilder(s.length());

        for (int i = 0; i < s.length(); i++) {
            if (s.charAt(i) == badfs.charAt(0)) {
                ret.append(SEPARATOR);
            } else {
                ret.append(s.charAt(i));
            }
        }

        return ret.toString();
    }

    protected Object nvl(@Nullable final Object a, final Object b) {
        if (a != null) {
            return a;
        }
        return b;
    }

    /**
     * @param dtg expected format yyyymmddhhmmss
     * @return yyyy-mm-dd/hh/(mm%10)
     */
    protected String datePath(@Nullable final String dtg) {
        if (dtg == null) {
            return TimeUtil.getDateAsPath(Instant.now());
        } else {
            return dtg.substring(0, 4) + "-" + // yyyy
                    dtg.substring(4, 6) + "-" + // mm
                    dtg.substring(6, 8) + "/" + // dd
                    dtg.substring(8, 10) + "/" + // HH
                    dtg.substring(10, 11) + "0"; // M0
        }
    }

    /**
     * Make a dot file name from the supplied path Ex: /path/to/file.txt --&gt; /path/to/.file.txt
     *
     * @param fullName the full path and filename of the file
     * @return the path as is but with a leading dot in the filename
     */
    public String makeDotFile(final String fullName) {
        final int fpos = fullName.lastIndexOf("/");
        final int rpos = fullName.lastIndexOf("\\");
        if (fpos == -1 && rpos == -1) {
            return "." + fullName;
        }
        final int pos = Math.max(fpos, rpos);
        return fullName.substring(0, pos + 1) + "." + fullName.substring(pos + 1);
    }

    /**
     * Get the file type from the metadata or the form string passed in
     *
     * @param bdo IBaseDataObject
     * @return the file type
     */
    public static String getFileType(final IBaseDataObject bdo) {
        return getAndPutFileType(bdo, null, null);
    }

    /**
     * Get the file type from the IBaseDataObject or the form string passed in
     *
     * @param bdo IBaseDataObject
     * @param metaData Optional map of metadata that might be modified.
     * @param formsArg Optional space separated string of current forms
     * @return the file type
     */
    public static String getAndPutFileType(final IBaseDataObject bdo, @Nullable final Map<String, String> metaData, final String formsArg) {
        String forms = formsArg;
        if (forms == null) {
            forms = bdo.getStringParameter(FileTypeCheckParameter.POPPED_FORMS.getFieldName());
            if (forms == null) {
                forms = "";
            }
        }

        String fileType;
        if (bdo.hasParameter(FileTypeCheckParameter.FILETYPE.getFieldName())) {
            fileType = bdo.getStringParameter(FileTypeCheckParameter.FILETYPE.getFieldName());
        } else if (bdo.hasParameter(FileTypeCheckParameter.FINAL_ID.getFieldName())) {
            fileType = bdo.getStringParameter(FileTypeCheckParameter.FINAL_ID.getFieldName());
            logger.debug("FINAL_ID FileType is ({})", fileType);
            if (metaData != null) {
                metaData.put(FileTypeCheckParameter.FILETYPE.getFieldName(), fileType);
            }
        } else {
            if (forms.contains(" ")) {
                fileType = forms.substring(0, forms.indexOf(" ")).trim();
                if (metaData != null) {
                    metaData.put(FileTypeCheckParameter.COMPLETE_FILETYPE.getFieldName(), forms);
                }
            } else {
                fileType = forms;
            }
            if (StringUtils.isEmpty(fileType)) {
                if (bdo.hasParameter(FileTypeCheckParameter.FONT_ENCODING.getFieldName())) {
                    fileType = TEXT;
                } else {
                    fileType = UNKNOWN;
                }
            }

            if (metaData != null) {
                metaData.put(FileTypeCheckParameter.FILETYPE.getFieldName(), fileType);
            }
        }

        if (UNKNOWN.equals(fileType) && forms.contains("MSWORD")) {
            fileType = "MSWORD_FRAGMENT";
        }

        if ("QUOTED-PRINTABLE".equals(fileType) || fileType.startsWith(PREFIXES_LANG) || fileType.startsWith("ENCODING(")) {
            fileType = TEXT;
        }

        return fileType;
    }

    /**
     * Parameters that are used to determine filetype in {@link #getAndPutFileType(IBaseDataObject, Map, String)}
     */
    public enum FileTypeCheckParameter {
        COMPLETE_FILETYPE("COMPLETE_FILETYPE"), FILETYPE("FILETYPE"), FINAL_ID("FINAL_ID"), FONT_ENCODING("FontEncoding"), POPPED_FORMS(
                "POPPED_FORMS");

        String fieldName;

        FileTypeCheckParameter(String fieldName) {
            this.fieldName = fieldName;
        }

        public String getFieldName() {
            return fieldName;
        }
    }

    /**
     * Extract the ID from the payload. The ID from the payload is specified in the cfg file. An ID = SHORTNAME will use the
     * shortname. An ID = AUTO_GENERATED_ID will use an auto gen uuid. If no id value is found, defaults to using an auto
     * generated id.
     *
     * @param d the payload
     * @param tld if a param is specified and cannot be found in the d method parameter, tries to get param from tld.
     * @return the id based the best ID available, shortname or auto gen id. If no id is found, defaults to auto gen.
     */
    public String getBestId(final IBaseDataObject d, @Nullable final IBaseDataObject tld) {

        for (final String s : this.idTokens) {
            if (AUTO_GENERATED_ID.equals(s)) {
                String parentAutoGeneratedId = null;
                if (tld != null) {
                    parentAutoGeneratedId = tld.getStringParameter(PARENT_AUTO_GENERATED_ID);
                }
                if (StringUtils.isBlank(parentAutoGeneratedId)) {
                    String uuid = getRandomUUID(d);
                    if (tld != null) {
                        tld.setParameter(PARENT_AUTO_GENERATED_ID, uuid);
                    }
                    if (!StringUtils.isBlank(uuid)) {
                        final String component = d.shortName();
                        final int pos = component.indexOf(Family.SEP);
                        if (pos > -1) {
                            uuid += component.substring(pos);
                        }
                        return uuid;
                    }
                }
                String uuid = null;
                // try getting param from the tld
                if (!StringUtils.isBlank(parentAutoGeneratedId) && (tld != null)) {
                    uuid = tld.getStringParameter(PARENT_AUTO_GENERATED_ID);
                }
                if (!StringUtils.isBlank(uuid)) {
                    final String component = d.shortName();
                    final int pos = component.indexOf(Family.SEP);
                    if (pos > -1) {
                        uuid += component.substring(pos);
                    }
                    d.setParameter(AUTO_GENERATED_ID, "yes");
                    return uuid;
                }

            }

            if (SHORTNAME.equals(s)) {
                final String shortName = d.shortName();

                // if shortname is not blank use it, if blank move on...
                if (!StringUtils.isBlank(shortName)) {
                    return shortName;
                }

                String path = d.getStringParameter(s);
                // try getting shortname from the tld
                if (StringUtils.isBlank(path) && (tld != null)) {
                    path = tld.getStringParameter(s);
                }
                if (!StringUtils.isBlank(path)) {
                    final String component = d.shortName();
                    final int pos = component.indexOf(Family.SEP);
                    if (pos > -1) {
                        path += component.substring(pos);
                    }
                    return path;
                }

            }
            // the param from the tld has priority over any child
            if ((tld != null) && !StringUtils.isBlank(tld.getStringParameter(s))) {
                String path = tld.getStringParameter(s);
                if (!StringUtils.isBlank(path)) {
                    final String component = d.shortName();
                    final int pos = component.indexOf(Family.SEP);
                    if (pos > -1) {
                        path += component.substring(pos);
                    }
                    return path;
                }
            }
            // if the param is not in the tld
            if (!StringUtils.isBlank(d.getStringParameter(s))) {
                return d.getStringParameter(s);
            }

        }

        // if nothing works, auto gen an id
        final String uuid = getRandomUUID(d);
        if (tld != null) {
            tld.setParameter(PARENT_AUTO_GENERATED_ID, uuid);
        }
        return uuid;
    }

    /**
     * Creates a UUID. Includes a prefix if specified.
     *
     * @param d Adds a parameter indicating an auto gen id.
     * @return uuid
     */
    private String getRandomUUID(final IBaseDataObject d) {
        String uuid = UUID.randomUUID().toString();
        // use the prefix
        if (!StringUtils.isBlank(this.autoGeneratedIdPrefix)) {
            uuid = uuid.substring(this.autoGeneratedIdPrefix.length());
            uuid = this.autoGeneratedIdPrefix + uuid;
        }
        d.setParameter(PARENT_AUTO_GENERATED_ID, uuid);
        d.setParameter(AUTO_GENERATED_ID, "yes");
        return uuid;
    }

    /**
     * Extract language or a default value
     *
     * @param d the payload
     * @return the language or the default value (never null)
     */
    public String getLanguage(final IBaseDataObject d) {
        String lang = d.getStringParameter("LANGUAGE");
        if (lang == null) {
            lang = "NONE";
        }
        return lang;
    }

    /**
     * Handle data
     *
     * @param d the data object in question
     * @return the Date when the event occurred
     */
    public Date getEventDate(final IBaseDataObject d, @Nullable final IBaseDataObject tld) {
        Date eventDate = extractEventDateFrom(d, false);
        if (eventDate == null && tld != null) {
            eventDate = extractEventDateFrom(tld, this.defaultEventDateToNow);
        }
        return eventDate;
    }

    public Date extractEventDateFrom(final IBaseDataObject d, final boolean lastResortDefault) {
        Date date = null;

        for (final String paramName : this.dateTokens) {
            final String value = d.getStringParameter(paramName);
            if (value != null) {
                try {
                    date = TimeUtil.getDateFromISO8601(value);
                    return date;
                } catch (DateTimeParseException ex) {
                    logger.debug("Cannot parse EventDate", ex);
                }
            }
        }

        // Default to current system time if last resort
        return lastResortDefault ? new Date() : null;
    }

    /**
     * Extracts from the provided {@link IBaseDataObject} the last file extension from each value in the "Original-Filename"
     * parameter, if that value contains "." before its last character. If one or more file extensions are extracted, the
     * IBaseDataObject's "FILEXT" parameter is set as the unique set of extracted file extensions, converted to lowercase.
     *
     * @param p IBaseDataObject to process
     *
     */
    void extractUniqueFileExtensions(IBaseDataObject p) {
        if (p.hasParameter(ORIGINAL_FILENAME)) {
            final Set<String> extensions = new HashSet<>();
            for (Object filename : p.getParameter(ORIGINAL_FILENAME)) {
                final String fn = (String) filename;
                if (StringUtils.isNotEmpty(fn) && fn.lastIndexOf('.') > -1) {
                    final int pos = fn.lastIndexOf('.') + 1;
                    if (pos < fn.length()) {
                        final String fext = fn.substring(pos).toLowerCase();
                        if (fext.length() > 0 && fext.length() <= this.maxFilextLen) {
                            extensions.add(fext);
                        }
                    }
                }
            }
            if (!extensions.isEmpty()) {
                p.setParameter("FILEXT", extensions);
            }
        }
    }

    /**
     * Process metadata before doing any output
     *
     * @param payloadList list of items being dropped off that may need initial metadata computations
     */
    public void processMetadata(final List<IBaseDataObject> payloadList) {

        // Keep track of parent's filetype to output
        // relies on the attachments being sorted
        final Map<String, String> parentTypes = new HashMap<>();
        final IBaseDataObject tld = payloadList.get(0);
        final List<String> extended_filetypes = new ArrayList<>();
        parentTypes.put("1", tld.getFileType());
        for (int i = 0; i < parentParams.size(); i++) {
            final String param = parentParams.get(i);
            if (tld.hasParameter(param)) {
                parentTypes.put("1" + param, tld.getStringParameter(param));
            }
        }

        for (final IBaseDataObject p : payloadList) {
            final int level = StringUtils.countMatches(p.shortName(), emissary.core.Family.SEP) + 1;
            // save specified metadata items for children to grab
            parentTypes.put("" + level, p.getFileType());

            extractUniqueFileExtensions(p);

            if (p.getStringParameter("EXTENDED_FILETYPE") == null) {
                extended_filetypes.clear();
                for (final Map.Entry<String, Collection<Object>> entry : p.getParameters().entrySet()) {
                    final String key = entry.getKey();
                    if (key != null && key.endsWith("_FILETYPE")) {
                        for (final Object value : entry.getValue()) {
                            final String vs = value.toString();
                            if (!extended_filetypes.contains(vs)) {
                                extended_filetypes.add(vs);
                            }
                        }
                    }
                }
                if (!extended_filetypes.isEmpty()) {
                    final StringBuilder extft = new StringBuilder(getFileType(p));
                    for (int j = 0; j < extended_filetypes.size(); j++) {
                        final String s = extended_filetypes.get(j);
                        extft.append("//").append(s);
                    }
                    p.setParameter("EXTENDED_FILETYPE", extft.toString());
                }
            }

            for (int j = 0; j < parentParams.size(); j++) {
                final String param = parentParams.get(j);
                if (p.hasParameter(param)) {
                    parentTypes.put("" + level + param, p.getStringParameter(param));
                } else {
                    // Must clear to keep my children from getting their uncle's value
                    parentTypes.remove("" + level + param);
                }

            }

            if (level > 1) {
                // then it might want some PARENT info

                final int parentLevel = level - 1;
                final String pType = parentTypes.get("" + parentLevel);
                if (pType != null && pType.length() > 0) {
                    p.setParameter("PARENT_FILETYPE", pType);
                } else {
                    p.setParameter("PARENT_FILETYPE", parentTypes.get("1"));
                }
                for (int j = 0; j < parentParams.size(); j++) {
                    final String param = parentParams.get(j);
                    int plvl = parentLevel;
                    while (plvl > 1 && !parentTypes.containsKey("" + plvl + param)) {
                        plvl--;
                    }
                    if (StringUtils.isNotBlank(parentTypes.get(plvl + param))) {
                        p.setParameter("PARENT_" + param, parentTypes.get(plvl + param));
                    }
                }
            }

            if (p.hasExtractedRecords()) {
                final List<IBaseDataObject> childObjList = p.getExtractedRecords();
                Collections.sort(childObjList, new ShortNameComparator());
                for (final IBaseDataObject child : childObjList) {
                    final int parentLevel = StringUtils.countMatches(child.shortName(), emissary.core.Family.SEP);
                    final String parentFileType = parentTypes.get("" + parentLevel);
                    if (parentFileType != null) {
                        child.setParameter("PARENT_FILETYPE", parentFileType);
                    }
                    for (int k = 0; k < parentParams.size(); k++) {
                        final String param = parentParams.get(k);
                        int plvl = parentLevel;
                        while (plvl > 1 && !parentTypes.containsKey("" + plvl + param)) {
                            plvl--;
                        }
                        if (StringUtils.isNotBlank(parentTypes.get(plvl + param))) {
                            child.setParameter("PARENT_" + param, parentTypes.get(plvl + param));
                        }
                    }
                }
            }
        }
    }
}
